<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  <meta http-equiv="X-UA-Compatible" content="IE=edge">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <meta name="author" content="PG-Strom Development Team">
  <link rel="shortcut icon" href="../img/favicon.ico">
  <title>Install - PG-Strom Manual</title>
  <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'>

  <link rel="stylesheet" href="../css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../css/theme_extra.css" type="text/css" />
  <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
  <link href="//fonts.googleapis.com/earlyaccess/notosansjp.css" rel="stylesheet">
  <link href="//fonts.googleapis.com/css?family=Open+Sans:600,800" rel="stylesheet">
  <link href="../custom.css" rel="stylesheet">
  
  <script>
    // Current page data
    var mkdocs_page_name = "Install";
    var mkdocs_page_input_path = "install.md";
    var mkdocs_page_url = null;
  </script>
  
  <script src="../js/jquery-2.1.1.min.js" defer></script>
  <script src="../js/modernizr-2.8.3.min.js" defer></script>
  <script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
  <script>hljs.initHighlightingOnLoad();</script> 
  
</head>

<body class="wy-body-for-nav" role="document">

  <div class="wy-grid-for-nav">

    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
      <div class="wy-side-nav-search">
        <a href=".." class="icon icon-home"> PG-Strom Manual</a>
        <div role="search">
  <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
  </form>
  [<a href="../ja/install/" style="color: #cccccc">Japanese</a> | <strong>English</strong>]
</div>
      </div>

      <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
	<ul class="current">
	  
          
            <li class="toctree-l1">
		
    <a class="" href="..">Home</a>
	    </li>
          
            <li class="toctree-l1 current">
		
    <a class="current" href="./">Install</a>
    <ul class="subnav">
            
    <li class="toctree-l2"><a href="#checklist">Checklist</a></li>
    

    <li class="toctree-l2"><a href="#os-installation">OS Installation</a></li>
    
        <ul>
        
            <li><a class="toctree-l3" href="#post-os-installation-configuration">Post OS Installation Configuration</a></li>
        
        </ul>
    

    <li class="toctree-l2"><a href="#cuda-toolkit-installation">CUDA Toolkit Installation</a></li>
    

    <li class="toctree-l2"><a href="#postgresql-installation">PostgreSQL Installation</a></li>
    

    <li class="toctree-l2"><a href="#pg-strom-installation">PG-Strom Installation</a></li>
    
        <ul>
        
            <li><a class="toctree-l3" href="#rpm-installation">RPM Installation</a></li>
        
            <li><a class="toctree-l3" href="#installation-from-the-source">Installation from the source</a></li>
        
            <li><a class="toctree-l3" href="#post-installation-setup">Post Installation Setup</a></li>
        
        </ul>
    

    <li class="toctree-l2"><a href="#nvme-strom-module">NVME-Strom module</a></li>
    
        <ul>
        
            <li><a class="toctree-l3" href="#getting-the-module-and-installation">Getting the module and installation</a></li>
        
            <li><a class="toctree-l3" href="#license-activation">License activation</a></li>
        
            <li><a class="toctree-l3" href="#kernel-module-parameters">Kernel module parameters</a></li>
        
        </ul>
    

    <li class="toctree-l2"><a href="#postgis-installation">PostGIS Installation</a></li>
    

    </ul>
	    </li>
          
            <li class="toctree-l1">
		
    <span class="caption-text">Tutorial</span>
    <ul class="subnav">
                <li class="">
                    
    <a class="" href="../operations/">Basic Operations</a>
                </li>
                <li class="">
                    
    <a class="" href="../sys_admin/">System Administration</a>
                </li>
                <li class="">
                    
    <a class="" href="../brin/">BRIN Index</a>
                </li>
                <li class="">
                    
    <a class="" href="../partition/">Partitioning</a>
                </li>
                <li class="">
                    
    <a class="" href="../postgis/">PostGIS</a>
                </li>
                <li class="">
                    
    <a class="" href="../troubles/">Trouble Shooting</a>
                </li>
    </ul>
	    </li>
          
            <li class="toctree-l1">
		
    <span class="caption-text">Advanced Features</span>
    <ul class="subnav">
                <li class="">
                    
    <a class="" href="../ssd2gpu/">GPUDirect SQL</a>
                </li>
                <li class="">
                    
    <a class="" href="../arrow_fdw/">Apache Arrow</a>
                </li>
                <li class="">
                    
    <a class="" href="../gstore_fdw/">GPU Memory Store</a>
                </li>
                <li class="">
                    
    <a class="" href="../python/">Python cooperation</a>
                </li>
    </ul>
	    </li>
          
            <li class="toctree-l1">
		
    <span class="caption-text">References</span>
    <ul class="subnav">
                <li class="">
                    
    <a class="" href="../ref_types/">Data Types</a>
                </li>
                <li class="">
                    
    <a class="" href="../ref_devfuncs/">Functions and Operators</a>
                </li>
                <li class="">
                    
    <a class="" href="../ref_sqlfuncs/">SQL Objects</a>
                </li>
                <li class="">
                    
    <a class="" href="../ref_params/">GUC Parameters</a>
                </li>
    </ul>
	    </li>
          
            <li class="toctree-l1">
		
    <a class="" href="../release_note/">Release Note</a>
	    </li>
          
        </ul>
      </div>
      &nbsp;
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
        <a href="..">PG-Strom Manual</a>
      </nav>

      
      <div class="wy-nav-content">
        <div class="rst-content">
          <div role="navigation" aria-label="breadcrumbs navigation">
  <ul class="wy-breadcrumbs">
    <li><a href="..">Docs</a> &raquo;</li>
    
      
    
    <li>Install</li>
    <li class="wy-breadcrumbs-aside">
      
    </li>
  </ul>
  <hr/>
</div>
          <div role="main">
            <div class="section">
              
                <p>This chapter introduces the steps to install PG-Strom.</p>
<h1 id="checklist">Checklist</h1>
<ul>
<li><strong>Server Hardware</strong><ul>
<li>It requires generic x86_64 hardware that can run Linux operating system supported by CUDA Toolkit. We have no special requirement for CPU, storage and network devices.</li>
<li><a href="https://github.com/heterodb/pg-strom/wiki/002:-HW-Validation-List">note002:HW Validation List</a> may help you to choose the hardware.</li>
<li>SSD-to-GPU Direct SQL Execution needs SSD devices which support NVMe specification, and to be installed under the same PCIe Root Complex where GPU is located on.</li>
</ul>
</li>
<li><strong>GPU Device</strong><ul>
<li>PG-Strom requires at least one GPU device on the system, which is supported by CUDA Toolkit, has computing capability 6.0 (Pascal generation) or later;</li>
<li><a href="https://github.com/heterodb/pg-strom/wiki/001:-GPU-Availability-Matrix">note001:GPU Availability Matrix</a> shows more detailed information. Check this list for the support status of SSD-to-GPU Direct SQL Execution.</li>
</ul>
</li>
<li><strong>Operating System</strong><ul>
<li>PG-Strom requires Linux operating system for x86_64 architecture, and its distribution supported by CUDA Toolkit. Our recommendation is Red Hat Enterprise Linux or CentOS version 7.x series.    - SSD-to-GPU Direct SQL Execution needs Red Hat Enterprise Linux or CentOS version 7.3 or later.</li>
</ul>
</li>
<li><strong>PostgreSQL</strong><ul>
<li>PG-Strom requires PostgreSQL version 9.6 or later. PostgreSQL v9.6 renew the custom-scan interface for CPU-parallel execution or <code>GROUP BY</code> planning, thus, it allows cooperation of custom-plans provides by extension modules.</li>
</ul>
</li>
<li><strong>CUDA Toolkit</strong><ul>
<li>PG-Strom requires CUDA Toolkit version 9.2 or later.</li>
<li>Some of CUDA Driver APIs used by PG-Strom internally are not included in the former versions.</li>
</ul>
</li>
</ul>
<h1 id="os-installation">OS Installation</h1>
<p>Choose a Linux distribution which is supported by CUDA Toolkit, then install the system according to the installation process of the distribution. <a href="https://developer.nvidia.com/">NVIDIA DEVELOPER ZONE</a> introduces the list of Linux distributions which are supported by CUDA Toolkit.</p>
<p>In case of Red Hat Enterprise Linux 7.x or CentOS 7.x series, choose "Minimal installation" as base environment, and also check the following add-ons.</p>
<ul>
<li>Debugging Tools</li>
<li>Development Tools</li>
</ul>
<h2 id="post-os-installation-configuration">Post OS Installation Configuration</h2>
<p>Next to the OS installation, a few additionsl configurations are required to install GPU-drivers and NVMe-Strom driver on the later steps.</p>
<h3 id="setup-epel-repository">Setup EPEL Repository</h3>
<p>Several software modules required by PG-Strom are distributed as a part of EPEL (Extra Packages for Enterprise Linux).
You need to add a repository definition of EPEL packages for yum system to obtain these software.</p>
<p>One of the package we will get from EPEL repository is DKMS (Dynamic Kernel Module Support). It is a framework to build Linux kernel module for the running Linux kernel on demand; used for NVIDIA's GPU driver or NVMe-Strom which is a kernel module to support SSD-to-GPU Direct SQL Execution.</p>
<p><code>epel-release</code> package provides the repository definition of EPEL.
You can obtain this package from the public FTP site of Fedora Project. Downloads the <code>epel-release-&lt;distribution version&gt;.noarch.rpm</code>, and install the package.
Once <code>epel-release</code> package gets installed, yum system configuration is updated to get software from the EPEL repository.</p>
<ul>
<li>Fedora Project Public FTP Site<ul>
<li><a href="https://dl.fedoraproject.org/pub/epel/7/x86_64/">https://dl.fedoraproject.org/pub/epel/7/x86_64/</a></li>
</ul>
</li>
</ul>
<div class="admonition tip">
<p class="admonition-title">Tip</p>
<p>Walk down the directory: <code>Packages</code> --&gt; <code>e</code>, from the above URL.</p>
</div>
<p>Install the <code>epel-release</code> package as follows.</p>
<pre><code>$ sudo yum install https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/e/epel-release-7-11.noarch.rpm
          :
================================================================================
 Package           Arch        Version     Repository                      Size
================================================================================
Installing:
 epel-release      noarch      7-11        /epel-release-7-11.noarch       24 k

Transaction Summary
================================================================================
Install  1 Package
          :
Installed:
  epel-release.noarch 0:7-11

Complete!
</code></pre>

<h3 id="heterodb-swdc-installation">HeteroDB-SWDC Installation</h3>
<p>PG-Strom and related packages are distributed from <a href="https://heterodb.github.io/swdc/">HeteroDB Software Distribution Center</a>.
You need to add a repository definition of HeteroDB-SWDC for you system to obtain these software.</p>
<p><code>heterodb-swdc</code> package provides the repository definition of HeteroDB-SWDC.
Access to the <a href="https://heterodb.github.io/swdc/">HeteroDB Software Distribution Center</a> using Web browser, download the <code>heterodb-swdc-1.0-1.el7.noarch.rpm</code> on top of the file list, then install this package.
Once heterodb-swdc package gets installed, yum system configuration is updated to get software from the HeteroDB-SWDC repository.</p>
<p>Install the <code>heterodb-swdc</code> package as follows.</p>
<pre><code>$ sudo yum install https://heterodb.github.io/swdc/yum/rhel7-x86_64/heterodb-swdc-1.0-1.el7.noarch.rpm
          :
================================================================================
 Package         Arch     Version       Repository                         Size
================================================================================
Installing:
 heterodb-swdc   noarch   1.0-1.el7     /heterodb-swdc-1.0-1.el7.noarch   2.4 k

Transaction Summary
================================================================================
Install  1 Package
          :
Installed:
  heterodb-swdc.noarch 0:1.0-1.el7

Complete!
</code></pre>

<h1 id="cuda-toolkit-installation">CUDA Toolkit Installation</h1>
<p>This section introduces the installation of CUDA Toolkit. If you already installed the latest CUDA Toolkit, you can skip this section.</p>
<p>NVIDIA offers two approach to install CUDA Toolkit; one is by self-extracting archive (called runfile), and the other is by RPM packages.
We recommend RPM installation because it allows simple software updates.</p>
<p>You can download the installation package for CUDA Toolkit from NVIDIA DEVELOPER ZONE. Choose your OS, architecture, distribution and version, then choose "rpm(network)" edition.</p>
<p><img alt="CUDA Toolkit download" src="../img/cuda-download.png" /></p>
<p>The "rpm(network)" edition contains only yum repositoty definition to distribute CUDA Toolkit. It is similar to the EPEL repository definition at the OS installation.
So, you needs to installa the related RPM packages over network after the resistoration of CUDA repository. Run the following command.</p>
<pre><code>$ sudo rpm -i cuda-repo-&lt;distribution&gt;-&lt;version&gt;.x86_64.rpm
$ sudo yum clean all
$ sudo yum install cuda --enablerepo=rhel-7-server-e4s-optional-rpms
 or
$ sudo yum install cuda 
</code></pre>

<p>Once installation completed successfully, CUDA Toolkit is deployed at <code>/usr/local/cuda</code>.</p>
<div class="admonition tip">
<p class="admonition-title">Tip</p>
<p>RHEL7 does not enable <code>rhel-7-server-e4s-optional-rpms</code> repository in the default. It distributes <code>vulkan-filesystem</code> packaged required by CUDA Toolkit installation. When you kick installation of CUDA Toolkit, edit <code>/etc/yum.repos.d/redhat.repo</code> to enable the repository, or use <code>--enablerepo</code> option of yum command to resolve dependency.</p>
</div>
<pre><code>$ ls /usr/local/cuda
bin     include  libnsight         nvml       samples  tools
doc     jre      libnvvp           nvvm       share    version.txt
extras  lib64    nsightee_plugins  pkgconfig  src
</code></pre>

<p>Once installation gets completed, ensure the system recognizes the GPU devices correctly.
<code>nvidia-smi</code> command shows GPU information installed on your system, as follows.</p>
<pre><code>$ nvidia-smi
Wed Feb 14 09:43:48 2018
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 387.26                 Driver Version: 387.26                    |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  Tesla V100-PCIE...  Off  | 00000000:02:00.0 Off |                    0 |
| N/A   41C    P0    37W / 250W |      0MiB / 16152MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+
</code></pre>

<div class="admonition tip">
<p class="admonition-title">Tip</p>
<p>If nouveau driver which conflicts to nvidia driver is loaded, system cannot load the nvidia driver immediately.
In this case, reboot the operating system after a configuration to disable the nouveau driver.
If CUDA Toolkit is installed by the runfile installer, it also disables the nouveau driver. Elsewhere, in case of RPM installation, do the following configuration.</p>
</div>
<p>To disable the nouveau driver, put the following configuration onto <code>/etc/modprobe.d/disable-nouveau.conf</code>, then run <code>dracut</code> command to apply them on the boot image of Linux kernel.</p>
<pre><code># cat &gt; /etc/modprobe.d/disable-nouveau.conf &lt;&lt;EOF
blacklist nouveau
options nouveau modeset=0
EOF
# dracut -f
</code></pre>

<h1 id="postgresql-installation">PostgreSQL Installation</h1>
<p>This section introduces PostgreSQL installation with RPM.
We don't introduce the installation steps from the source because there are many documents for this approach, and there are also various options for the <code>./configure</code> script.</p>
<p>PostgreSQL is also distributed in the packages of Linux distributions, however, it is not the latest one, and often older than the version which supports PG-Strom. For example, Red Hat Enterprise Linux 7.x or CentOS 7.x distributes PostgreSQL v9.2.x series. This version had been EOL by the PostgreSQL community.</p>
<p>PostgreSQL Global Development Group provides yum repository to distribute the latest PostgreSQL and related packages.
Like the configuration of EPEL, you can install a small package to set up yum repository, then install PostgreSQL and related software.</p>
<p>Here is the list of yum repository definition: <a href="http://yum.postgresql.org/repopackages.php">http://yum.postgresql.org/repopackages.php</a>.</p>
<p>Repository definitions are per PostgreSQL major version and Linux distribution. You need to choose the one for your Linux distribution, and for PostgreSQL v10 or later.</p>
<p>All you need to install are yum repository definition, and PostgreSQL packages. If you choose PostgreSQL v10, the pakages below are required to install PG-Strom.</p>
<ul>
<li>postgresql10-devel</li>
<li>postgresql10-server</li>
</ul>
<pre><code>$ sudo yum install -y https://download.postgresql.org/pub/repos/yum/10/redhat/rhel-7-x86_64/pgdg-redhat10-10-2.noarch.rpm
$ sudo yum install -y postgresql10-server postgresql10-devel
          :
================================================================================
 Package                  Arch        Version                 Repository   Size
================================================================================
Installing:
 postgresql10-devel       x86_64      10.2-1PGDG.rhel7        pgdg10      2.0 M
 postgresql10-server      x86_64      10.2-1PGDG.rhel7        pgdg10      4.4 M
Installing for dependencies:
 postgresql10             x86_64      10.2-1PGDG.rhel7        pgdg10      1.5 M
 postgresql10-libs        x86_64      10.2-1PGDG.rhel7        pgdg10      354 k

Transaction Summary
================================================================================
Install  2 Packages (+2 Dependent packages)
          :
Installed:
  postgresql10-devel.x86_64 0:10.2-1PGDG.rhel7
  postgresql10-server.x86_64 0:10.2-1PGDG.rhel7

Dependency Installed:
  postgresql10.x86_64 0:10.2-1PGDG.rhel7
  postgresql10-libs.x86_64 0:10.2-1PGDG.rhel7

Complete!
</code></pre>

<div class="admonition note">
<p class="admonition-title">Note</p>
<p>On the Red Hat Enterprise Linux 8 and CentOS 8, the package name <code>postgresql</code> conflicts to the default one at the distribution, then unable to install the package from PGDG. Disable the <code>postgresql</code> package by the distribution, using the command below.</p>
</div>
<pre><code># dnf -y module disable postgresql
   :
Complete!
</code></pre>

<p>The RPM packages provided by PostgreSQL Global Development Group installs software under the <code>/usr/pgsql-&lt;version&gt;</code> directory, so you may pay attention whether the PATH environment variable is configured appropriately.</p>
<p><code>postgresql-alternative</code> package set up symbolic links to the related commands under <code>/usr/local/bin</code>, so allows to simplify the operations. Also, it enables to switch target version using <code>alternatives</code> command even if multiple version of PostgreSQL.</p>
<pre><code>$ sudo yum install postgresql-alternatives
          :
Resolving Dependencies
--&gt; Running transaction check
---&gt; Package postgresql-alternatives.noarch 0:1.0-1.el7 will be installed
--&gt; Finished Dependency Resolution

Dependencies Resolved
          :
================================================================================
 Package                      Arch        Version           Repository     Size
================================================================================
Installing:
 postgresql-alternatives      noarch      1.0-1.el7         heterodb      9.2 k

Transaction Summary
================================================================================
          :
Installed:
  postgresql-alternatives.noarch 0:1.0-1.el7

Complete!
</code></pre>

<h1 id="pg-strom-installation">PG-Strom Installation</h1>
<h2 id="rpm-installation">RPM Installation</h2>
<p>PG-Strom and related packages are distributed from <a href="https://heterodb.github.io/swdc/">HeteroDB Software Distribution Center</a>.
If you repository definition has been added, not many tasks are needed.</p>
<p>We provide individual RPM packages of PG-Strom for each base PostgreSQL version. <code>pg_strom-PG96</code> package is built for PostgreSQL 9.6, and <code>pg_strom-PG10</code> is also built for PostgreSQL v10.</p>
<pre><code>$ sudo yum install pg_strom-PG10
          :
================================================================================
 Package              Arch          Version               Repository       Size
================================================================================
Installing:
 pg_strom-PG10        x86_64        1.9-180301.el7        heterodb        320 k

Transaction Summary
================================================================================
          :
Installed:
  pg_strom-PG10.x86_64 0:1.9-180301.el7

Complete!
</code></pre>

<p>That's all for package installation.</p>
<h2 id="installation-from-the-source">Installation from the source</h2>
<p>For developers, we also introduces the steps to build and install PG-Strom from the source code.</p>
<h3 id="getting-the-source-code">Getting the source code</h3>
<p>Like RPM packages, you can download tarball of the source code from <a href="https://heterodb.github.io/swdc/">HeteroDB Software Distribution Center</a>.
On the other hands, here is a certain time-lags to release the tarball, it may be preferable to checkout the master branch of <a href="https://github.com/heterodb/pg-strom">PG-Strom on GitHub</a> to use the latest development branch.</p>
<pre><code>$ git clone https://github.com/heterodb/pg-strom.git
Cloning into 'pg-strom'...
remote: Counting objects: 13797, done.
remote: Compressing objects: 100% (215/215), done.
remote: Total 13797 (delta 208), reused 339 (delta 167), pack-reused 13400
Receiving objects: 100% (13797/13797), 11.81 MiB | 1.76 MiB/s, done.
Resolving deltas: 100% (10504/10504), done.
</code></pre>

<h3 id="building-the-pg-strom">Building the PG-Strom</h3>
<p>Configuration to build PG-Strom must match to the target PostgreSQL strictly. For example, if a particular <code>strcut</code> has inconsistent layout by the configuration at build, it may lead problematic bugs; not easy to find out.
Thus, not to have inconsistency, PG-Strom does not have own configure script, but references the build configuration of PostgreSQL using <code>pg_config</code> command.</p>
<p>If PATH environment variable is set to the <code>pg_config</code> command of the target PostgreSQL, run <code>make</code> and <code>make install</code>.
Elsewhere, give <code>PG_CONFIG=...</code> parameter on <code>make</code> command to tell the full path of the <code>pg_config</code> command.</p>
<pre><code>$ cd pg-strom
$ make PG_CONFIG=/usr/pgsql-10/bin/pg_config
$ sudo make install PG_CONFIG=/usr/pgsql-10/bin/pg_config
</code></pre>

<h2 id="post-installation-setup">Post Installation Setup</h2>
<h3 id="creation-of-database-cluster">Creation of database cluster</h3>
<p>Database cluster is not constructed yet, run <code>initdb</code> command to set up initial database of PostgreSQL.</p>
<p>The default path of the database cluster on RPM installation is <code>/var/lib/pgsql/&lt;version number&gt;/data</code>.
If you install <code>postgresql-alternatives</code> package, this default path can be referenced by <code>/var/lib/pgdata</code> regardless of the PostgreSQL version.</p>
<pre><code>$ sudo su - postgres
$ initdb -D /var/lib/pgdata/
The files belonging to this database system will be owned by user &quot;postgres&quot;.
This user must also own the server process.

The database cluster will be initialized with locale &quot;en_US.UTF-8&quot;.
The default database encoding has accordingly been set to &quot;UTF8&quot;.
The default text search configuration will be set to &quot;english&quot;.

Data page checksums are disabled.

fixing permissions on existing directory /var/lib/pgdata ... ok
creating subdirectories ... ok
selecting default max_connections ... 100
selecting default shared_buffers ... 128MB
selecting dynamic shared memory implementation ... posix
creating configuration files ... ok
running bootstrap script ... ok
performing post-bootstrap initialization ... ok
syncing data to disk ... ok

WARNING: enabling &quot;trust&quot; authentication for local connections
You can change this by editing pg_hba.conf or using the option -A, or
--auth-local and --auth-host, the next time you run initdb.

Success. You can now start the database server using:

    pg_ctl -D /var/lib/pgdata/ -l logfile start
</code></pre>

<h3 id="setup-postgresqlconf">Setup postgresql.conf</h3>
<p>Next, edit <code>postgresql.conf</code> which is a configuration file of PostgreSQL.
The parameters below should be edited at least to work PG-Strom.
Investigate other parameters according to usage of the system and expected workloads.</p>
<ul>
<li><strong>shared_preload_libraries</strong><ul>
<li>PG-Strom module must be loaded on startup of the postmaster process by the <code>shared_preload_libraries</code>. Unable to load it on demand. Therefore, you must add the configuration below.</li>
<li><code>shared_preload_libraries = '$libdir/pg_strom'</code></li>
</ul>
</li>
<li><strong>max_worker_processes</strong><ul>
<li>PG-Strom internally uses several background workers, so the default configuration (= 8) is too small for other usage. So, we recommand to expand the variable for a certain margin.</li>
<li><code>max_worker_processes = 100</code></li>
</ul>
</li>
<li><strong>shared_buffers</strong><ul>
<li>Although it depends on the workloads, the initial configuration of <code>shared_buffers</code> is too small for the data size where PG-Strom tries to work, thus storage workloads restricts the entire performance, and may be unable to work GPU efficiently.</li>
<li>So, we recommend to expand the variable for a certain margin.</li>
<li><code>shared_buffers = 10GB</code></li>
<li>Please consider to apply <strong>SSD-to-GPU Direct SQL Execution</strong> to process larger than system's physical RAM size.</li>
</ul>
</li>
<li><strong>work_mem</strong><ul>
<li>Although it depends on the workloads, the initial configuration of <code>work_mem</code> is too small to choose the optimal query execution plan on analytic queries.</li>
<li>An typical example is, disk-based merge sort may be chosen instead of the in-memory quick-sorting.</li>
<li>So, we recommend to expand the variable for a certain margin.</li>
<li><code>work_mem = 1GB</code></li>
</ul>
</li>
</ul>
<h3 id="expand-os-resource-limits">Expand OS resource limits</h3>
<p>SSD-to-GPU Direct SQL especially tries to open many files simultaneously, so resource limit for number of file descriptors per process should be expanded.</p>
<p>Also, we recommend not to limit core file size to generate core dump of PostgreSQL certainly on system crash.</p>
<p>If PostgreSQL service is launched by systemd, you can put the configurations of resource limit at <code>/etc/systemd/system/postgresql-XX.service.d/pg_strom.conf</code>.</p>
<p>RPM installation setups the configuration below by the default.</p>
<p>It comments out configuration to the environment variable <code>CUDA_ENABLE_COREDUMP_ON_EXCEPTION</code>. This is a developer option that enables to generate GPU's core dump on any CUDA/GPU level errors, if enabled. See <a href="https://docs.nvidia.com/cuda/cuda-gdb/index.html#gpu-coredump">CUDA-GDB:GPU core dump support</a> for more details.</p>
<pre><code>[Service]
LimitNOFILE=65536
LimitCORE=infinity
#Environment=CUDA_ENABLE_COREDUMP_ON_EXCEPTION=1
</code></pre>

<h3 id="start-postgresql">Start PostgreSQL</h3>
<p>Start PostgreSQL service.</p>
<p>If PG-Strom is set up appropriately, it writes out log message which shows PG-Strom recognized GPU devices.
The example below recognized the Tesla V100(PCIe; 16GB edition) device.</p>
<pre><code># systemctl start postgresql-10
# systemctl status -l postgresql-10
* postgresql-10.service - PostgreSQL 10 database server
   Loaded: loaded (/usr/lib/systemd/system/postgresql-10.service; disabled; vendor preset: disabled)
   Active: active (running) since Sat 2018-03-03 15:45:23 JST; 2min 21s ago
     Docs: https://www.postgresql.org/docs/10/static/
  Process: 24851 ExecStartPre=/usr/pgsql-10/bin/postgresql-10-check-db-dir ${PGDATA} (code=exited, status=0/SUCCESS)
 Main PID: 24858 (postmaster)
   CGroup: /system.slice/postgresql-10.service
           |-24858 /usr/pgsql-10/bin/postmaster -D /var/lib/pgsql/10/data/
           |-24890 postgres: logger process
           |-24892 postgres: bgworker: PG-Strom GPU memory keeper
           |-24896 postgres: checkpointer process
           |-24897 postgres: writer process
           |-24898 postgres: wal writer process
           |-24899 postgres: autovacuum launcher process
           |-24900 postgres: stats collector process
           |-24901 postgres: bgworker: PG-Strom ccache-builder2
           |-24902 postgres: bgworker: PG-Strom ccache-builder1
           `-24903 postgres: bgworker: logical replication launcher

Mar 03 15:45:19 saba.heterodb.com postmaster[24858]: 2018-03-03 15:45:19.195 JST [24858] HINT:  Run 'nvidia-cuda-mps-control -d', then start server process. Check 'man nvidia-cuda-mps-control' for more details.
Mar 03 15:45:20 saba.heterodb.com postmaster[24858]: 2018-03-03 15:45:20.509 JST [24858] LOG:  PG-Strom: GPU0 Tesla V100-PCIE-16GB (5120 CUDA cores; 1380MHz, L2 6144kB), RAM 15.78GB (4096bits, 856MHz), CC 7.0
Mar 03 15:45:20 saba.heterodb.com postmaster[24858]: 2018-03-03 15:45:20.510 JST [24858] LOG:  NVRTC - CUDA Runtime Compilation vertion 9.1
Mar 03 15:45:23 saba.heterodb.com postmaster[24858]: 2018-03-03 15:45:23.378 JST [24858] LOG:  listening on IPv6 address &quot;::1&quot;, port 5432
Mar 03 15:45:23 saba.heterodb.com postmaster[24858]: 2018-03-03 15:45:23.378 JST [24858] LOG:  listening on IPv4 address &quot;127.0.0.1&quot;, port 5432
Mar 03 15:45:23 saba.heterodb.com postmaster[24858]: 2018-03-03 15:45:23.442 JST [24858] LOG:  listening on Unix socket &quot;/var/run/postgresql/.s.PGSQL.5432&quot;
Mar 03 15:45:23 saba.heterodb.com postmaster[24858]: 2018-03-03 15:45:23.492 JST [24858] LOG:  listening on Unix socket &quot;/tmp/.s.PGSQL.5432&quot;
Mar 03 15:45:23 saba.heterodb.com postmaster[24858]: 2018-03-03 15:45:23.527 JST [24858] LOG:  redirecting log output to logging collector process
Mar 03 15:45:23 saba.heterodb.com postmaster[24858]: 2018-03-03 15:45:23.527 JST [24858] HINT:  Future log output will appear in directory &quot;log&quot;.
Mar 03 15:45:23 saba.heterodb.com systemd[1]: Started PostgreSQL 10 database server.
</code></pre>

<h3 id="creation-of-pg-strom-related-objects">Creation of PG-Strom related objects</h3>
<p>At the last, create database objects related to PG-Strom, like SQL functions.
This steps are packaged using EXTENSION feature of PostgreSQL. So, all you needs to run is <code>CREATE EXTENSION</code> on the SQL command line.</p>
<p>Please note that this step is needed for each new database.
If you want PG-Strom is pre-configured on new database creation, you can create PG-Strom extension on the <code>template1</code> database, its configuration will be copied to the new database on <code>CREATE DATABASE</code> command.</p>
<pre><code>$ psql postgres -U postgres
psql (10.2)
Type &quot;help&quot; for help.

postgres=# CREATE EXTENSION pg_strom ;
CREATE EXTENSION
</code></pre>

<p>That's all for the installation.</p>
<h1 id="nvme-strom-module">NVME-Strom module</h1>
<p>This section also introduces NVME-Strom Linux kernel module which is closely cooperating with core features of PG-Strom like SSD-to-GPU Direct SQL Execution, even if it is an independent software module.</p>
<h2 id="getting-the-module-and-installation">Getting the module and installation</h2>
<p>Like other PG-Strom related modules, NVME-Strom is distributed at the (https://heterodb.github.io/swdc/)[HeteroDB Software Distribution Center] as a free software. In other words, it is not an open source software.</p>
<p>If your system already setup <code>heterodb-swdc</code> package, <code>yum install</code> command downloads the RPM file and install the <code>nvme_strom</code> package.</p>
<pre><code>$ sudo yum install nvme_strom
Loaded plugins: fastestmirror
Loading mirror speeds from cached hostfile
 * base: mirrors.cat.net
 * epel: ftp.iij.ad.jp
 * extras: mirrors.cat.net
 * ius: mirrors.kernel.org
 * updates: mirrors.cat.net
Resolving Dependencies
--&gt; Running transaction check
---&gt; Package nvme_strom.x86_64 0:1.3-1.el7 will be installed
--&gt; Finished Dependency Resolution

Dependencies Resolved

================================================================================
 Package             Arch            Version            Repository         Size
================================================================================
Installing:
 nvme_strom          x86_64          1.3-1.el7          heterodb          273 k

Transaction Summary
================================================================================
Install  1 Package

Total download size: 273 k
Installed size: 1.5 M
Is this ok [y/d/N]: y
Downloading packages:
No Presto metadata available for heterodb
nvme_strom-1.3-1.el7.x86_64.rpm                            | 273 kB   00:00
Running transaction check
Running transaction test
Transaction test succeeded
Running transaction
  Installing : nvme_strom-1.3-1.el7.x86_64                                  1/1
  :
&lt;snip&gt;
  :
DKMS: install completed.
  Verifying  : nvme_strom-1.3-1.el7.x86_64                                  1/1

Installed:
  nvme_strom.x86_64 0:1.3-1.el7

Complete!
</code></pre>

<h2 id="license-activation">License activation</h2>
<p>License activation is needed to use all the features of NVME-Strom module, provided by HeteroDB,Inc. You can operate the system without license, but features below are restricted.
- Multiple GPUs support
- Striping support (md-raid0) at SSD-to-GPU Direct SQL</p>
<p>You can obtain a license file, like as a plain text below, from HeteroDB,Inc.</p>
<pre><code>IAgIVdKxhe+BSer3Y67jQW0+uTzYh00K6WOSH7xQ26Qcw8aeUNYqJB9YcKJTJb+QQhjmUeQpUnboNxVwLCd3HFuLXeBWMKp11/BgG0FSrkUWu/ZCtDtw0F1hEIUY7m767zAGV8y+i7BuNXGJFvRlAkxdVO3/K47ocIgoVkuzBfLvN/h9LffOydUnHPzrFHfLc0r3nNNgtyTrfvoZiXegkGM9GBTAKyq8uWu/OGonh9ybzVKOgofhDLk0rVbLohOXDhMlwDl2oMGIr83tIpCWG+BGE+TDwsJ4n71Sv6n4bi/ZBXBS498qShNHDGrbz6cNcDVBa+EuZc6HzZoF6UrljEcl=
----
VERSION:2
SERIAL_NR:HDB-TRIAL
ISSUED_AT:2019-05-09
EXPIRED_AT:2019-06-08
GPU_UUID:GPU-a137b1df-53c9-197f-2801-f2dccaf9d42f
</code></pre>

<p>Copy the license file to <code>/etc/heterodb.license</code>, then restart PostgreSQL.</p>
<p>The startup log messages of PostgreSQL dumps the license information, and it tells us the license activation is successfully done.</p>
<pre><code>$ pg_ctl restart
   :
LOG:  PG-Strom version 2.2 built for PostgreSQL 11
LOG:  PG-Strom: GPU0 Tesla P40 (3840 CUDA cores; 1531MHz, L2 3072kB), RAM 22.38GB (384bits, 3.45GHz), CC 6.1
   :
LOG:  HeteroDB License: { &quot;version&quot; : 2, &quot;serial_nr&quot; : &quot;HDB-TRIAL&quot;, &quot;issued_at&quot; : &quot;9-May-2019&quot;, &quot;expired_at&quot; : &quot;8-Jun-2019&quot;, &quot;gpus&quot; : [ { &quot;uuid&quot; : &quot;GPU-a137b1df-53c9-197f-2801-f2dccaf9d42f&quot;, &quot;pci_id&quot; : &quot;0000:02:00.0&quot; } ] }
LOG:  listening on IPv6 address &quot;::1&quot;, port 5432
LOG:  listening on IPv4 address &quot;127.0.0.1&quot;, port 5432
    :
</code></pre>

<h2 id="kernel-module-parameters">Kernel module parameters</h2>
<p>NVME-Strom Linux kernel module has some parameters.</p>
<table>
<thead>
<tr>
<th align="center">Parameter</th>
<th align="center">Type</th>
<th align="center">Default</th>
<th align="center">Description</th>
</tr>
</thead>
<tbody>
<tr>
<td align="center"><code>verbose</code></td>
<td align="center"><code>int</code></td>
<td align="center"><code>0</code></td>
<td align="center">Enables detailed debug output</td>
</tr>
<tr>
<td align="center"><code>fast_ssd_mode</code></td>
<td align="center"><code>int</code></td>
<td align="center"><code>0</code></td>
<td align="center">Operating mode for fast NVME-SSD</td>
</tr>
<tr>
<td align="center"><code>p2p_dma_max_depth</code></td>
<td align="center"><code>int</code></td>
<td align="center"><code>1024</code></td>
<td align="center">Maximum number of asynchronous P2P DMA request can be enqueued on the I/O-queue of NVME device</td>
</tr>
<tr>
<td align="center"><code>p2p_dma_max_unitsz</code></td>
<td align="center"><code>int</code></td>
<td align="center"><code>256</code></td>
<td align="center">Maximum length of data blocks, in kB, to be read by a single P2P DMA request at once</td>
</tr>
</tbody>
</table>
<p>Here is an extra explanation for <code>fast_ssd_mode</code> parameter.</p>
<p>When NVME-Strom Linux kernel module get a request for SSD-to-GPU direct data transfer, first of all, it checks whether the required data blocks are caches on page-caches of operating system.
If <code>fast_ssd_mode</code> is <code>0</code>, NVME-Strom once writes back page caches of the required data blocks to the userspace buffer of the caller, then indicates application to invoke normal host--&gt;device data transfer by CUDA API. It is suitable for non-fast NVME-SSDs such as PCIe x4 grade.</p>
<p>On the other hands, SSD-to-GPU direct data transfer may be faster, if you use PCIe x8 grade fast NVME-SSD or use multiple SSDs in striping mode, than normal host--&gt;device data transfer after the buffer copy. If <code>fast_ssd_mode</code> is not <code>0</code>, NVME-Strom kicks SSD-to-GPU direct data transfer regardless of the page cache state.</p>
<p>However, it shall never kicks SSD-to-GPU direct data transfer if page cache is dirty.</p>
<p>Here is an extra explanation for <code>p2p_dma_max_depth</code> parameter.</p>
<p>NVME-Strom Linux kernel module makes DMA requests for SSD-to-GPU direct data transfer, then enqueues them to I/O-queue of the source NVME devices.
When asynchronous DMA requests are enqueued more than the capacity of NVME devices, latency of individual DMA requests become terrible because NVME-SSD controler processes the DMA requests in order of arrival. (On the other hands, it maximizes the throughput because NVME-SSD controler receives DMA requests continuously.)
If turn-around time of the DMA requests are too large, it may be wrongly considered as errors, then can lead timeout of I/O request and return an error status. Thus, it makes no sense to enqueue more DMA requests to the I/O-queue more than the reasonable amount of pending requests for full usage of NVME devices.</p>
<p><code>p2p_dma_max_depth</code> parameter controls number of asynchronous P2P DMA requests that can be enqueued at once per NVME device. If application tries to enqueue DMA requests more than the configuration, the caller thread will block until completion of the running DMA. So, it enables to avoid unintentional high-load of NVME devices.</p>
<h1 id="postgis-installation">PostGIS Installation</h1>
<p>PG-Strom supports execution of a part of PostGIS functions on GPU devices.
This section introduces the steps to install PostGIS module. Skip it on your demand.</p>
<p>PostGIS module can be installed from the yum repository by PostgreSQL Global Development Group, like PostgreSQL itself.
The example below shows the command to install PostGIS v3.0 built for PostgreSQL v12.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>CentOS 8 initial configuration does not enable the repository that delivers some libraries required by PostgreSQL,
add <code>--enablerepo=PowerTools</code> on the <code>dnf</code> command to activate PowerTools repository.</p>
</div>
<pre><code># dnf install postgis30_12 --enablerepo=PowerTools
Dependencies resolved.
====================================================================================================
 Package                           Architecture  Version                   Repository          Size
====================================================================================================
Installing:
 postgis30_12                      x86_64        3.0.1-5.rhel8             pgdg12             4.7 M
Installing dependencies:
 blas                              x86_64        3.8.0-8.el8               AppStream          429 k
   :                                  :                 :                     :                :
Complete!
</code></pre>

<p>Start PostgreSQL server after the initial setup of database cluster, then run <code>CREATE EXTENSION</code> command from SQL client to define geometry data type and SQL functions for geoanalytics.</p>
<pre><code>postgres=# CREATE EXTENSION postgis;
CREATE EXTENSION
</code></pre>
              
            </div>
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="../operations/" class="btn btn-neutral float-right" title="Basic Operations">Next <span class="icon icon-circle-arrow-right"></span></a>
      
      
        <a href=".." class="btn btn-neutral" title="Home"><span class="icon icon-circle-arrow-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <!-- Copyright etc -->
    
  </div>

  Built with <a href="http://www.mkdocs.org">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
      
        </div>
      </div>

    </section>

  </div>

  <div class="rst-versions" role="note" style="cursor: pointer">
    <span class="rst-current-version" data-toggle="rst-current-version">
      
      
        <span><a href=".." style="color: #fcfcfc;">&laquo; Previous</a></span>
      
      
        <span style="margin-left: 15px"><a href="../operations/" style="color: #fcfcfc">Next &raquo;</a></span>
      
    </span>
</div>
    <script>var base_url = '..';</script>
    <script src="../js/theme.js" defer></script>
      <script src="../search/main.js" defer></script>

</body>
</html>
